from select import select
from get_data import Content_url
from turn_to_md5 import To_md5
from sava_json_file import Save_as_Json
from turn_to_pdf import To_pdf
from concurrent.futures import ThreadPoolExecutor
from to_excel import To_excle
import random
import requests
import winsound
import os
import json
import time
import redis    # 导入redis 模块 
class Crawl:
    def __init__(self):
        self.md5_list = [] #创建存放MD5的空列表
        self.data_list = []
        self.data_dict = {}
        self.pool = redis.ConnectionPool(host='localhost', port=6379, decode_responses=True)   #redis连接池,避免每次建立、释放连接的开销。
        self.r = redis.Redis(host='localhost', port=6379, decode_responses=True) 
        self.id_list = []
        
    def request_post(self):
        for page in range(0,1):   #翻页处理
            start = Content_url('https://gs.amac.org.cn/amac-infodisc/api/pof/manager/query?rand=0.16468704311294613&page=%s&size=10'%str(page))
            data = start.get_content_url(str(page),self.data_dict,self.data_list)
        print(data)
        return data
    def save_to_redis(self,data):
        for i in data:
            content_url = i['url']
            turn = To_md5(content_url)      #实例化对象
            md5_code = turn.get_md5_value()    #将每个content_url都解码成MD5，作为键
        #   创建一个名为pdf_task的列表，将MD5和content_url键值对以列表的形式整合，再将其转成字符串作为列表的元素进行存储
            self.r.lpush('pdf_task',json.dumps([md5_code,content_url])) 
            print(self.r.llen('pdf_task'))
    def pick_redis_topdf(self):
        all_data_list = self.r.lrange('pdf_task',0,9)
        # print(all_data_list)
        for index, value in enumerate(all_data_list):
            data_pack = eval(value)       #value为字符串，需将其转为列表进行索引取值
            pdf = To_pdf(data_pack[1])
            pdf_file_path = './All_pdf/%s'%(data_pack[0] + '.pdf')
            pdf.url_to_pdf(pdf_file_path)
            self.r.lpop("pdf_task")
            print(self.r.llen('pdf_task'))
            
              
        
if __name__ == '__main__':
    start = Crawl()
    data = start.request_post()   #数据列表，字典为元素
    start.save_to_redis(data)
    start.pick_redis_topdf()        #从redis里面取出content_url
    
    










# if __name__ == '__main__':
#     md5_list = []       #创建存放MD5的空列表
#     data_list = []
#     data_dict = {}
#     pool = redis.ConnectionPool(host='localhost', port=6379, decode_responses=True)   #redis连接池,避免每次建立、释放连接的开销。
#     r = redis.Redis(host='localhost', port=6379, decode_responses=True) 

#     id_list = []
#     headers = {
#             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36',   
#     }
    
#     for page in range(0,1):   #翻页处理
#         start = Content_url('https://gs.amac.org.cn/amac-infodisc/api/pof/manager/query?rand=0.16468704311294613&page=%s&size=10'%str(page))
#         data = start.get_content_url(str(page),data_dict,data_list)
#     print(data)
#     for i in data:
#         content_url = i['url']
#         turn = To_md5(content_url)      #实例化对象
#         md5_code = turn.get_md5_value()    #将每个content_url都解码成MD5，作为键
#         #   创建一个名为pdf_task的列表，将MD5和content_url键值对以列表的形式整合，再将其转成字符串作为列表的元素进行存储
#         r.lpush('pdf_task',json.dumps([md5_code,content_url])) 
#         print(r.llen('pdf_task'))
    
#     all_data_list = r.lrange('pdf_task',0,9)
#     for index, value in enumerate(all_data_list):
#         data_pack = eval(value)       #value为字符串，需将其转为列表进行索引取值
        
#         pdf = To_pdf(data_pack[1])
#         pdf_file_path = './All_pdf/%s'%(data_pack[0] + '.pdf')
#         pdf_file = pdf.url_to_pdf(pdf_file_path)
   
#         delete = r.lpop("pdf_task")
#         print(delete)   
